{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "df063bdd-7d52-4731-bb79-234b487741ff",
   "metadata": {},
   "outputs": [],
   "source": [
    "#!/usr/bin/env python\n",
    "# coding: utf-8"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "bf93bff4-de25-4bb1-9105-b6dcfe0d0683",
   "metadata": {},
   "outputs": [],
   "source": [
    "# -*- coding: utf-8 -*- \n",
    "from hdfs import Client\n",
    "import sys\n",
    "import random\n",
    "#reload(sys)\n",
    "#sys.setdefaultencoding(\"utf-8\")\n",
    "\n",
    "import pandas as pd\n",
    "import logging\n",
    "import os\n",
    "import io\n",
    "import glob\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import defaultdict\n",
    "from sklearn.metrics import roc_auc_score\n",
    "import lightgbm as lgb\n",
    "import gc\n",
    "pd.set_option('display.min_rows',None)\n",
    "from sklearn.metrics import roc_curve,auc,roc_auc_score\n",
    "from sklearn.model_selection import train_test_split\n",
    "from tqdm import tqdm_notebook\n",
    "import time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "18393f56-b66c-4b1d-9493-8d489c6359ba",
   "metadata": {},
   "outputs": [],
   "source": [
    "def isNan_2(a):\n",
    "    return a != a\n",
    "\n",
    "\n",
    "def reduce_mem_usage(df, verbose=True):\n",
    "    numerics = ['int16', 'int32', 'int64', 'float16', 'float32', 'float64']\n",
    "    start_mem = df.memory_usage().sum() / 1024**2    \n",
    "    cols_ = [col for col in list(df) if col not in ['cid', 'vid']]\n",
    "    for col in cols_:\n",
    "        col_type = df[col].dtypes\n",
    "        if col_type in numerics:\n",
    "            c_min = df[col].min()\n",
    "            c_max = df[col].max()\n",
    "            if str(col_type)[:3] == 'int':\n",
    "                if c_min > np.iinfo(np.int8).min and c_max < np.iinfo(np.int8).max:\n",
    "                    df[col] = df[col].astype(np.int8)\n",
    "                elif c_min > np.iinfo(np.int16).min and c_max < np.iinfo(np.int16).max:\n",
    "                       df[col] = df[col].astype(np.int16)\n",
    "                elif c_min > np.iinfo(np.int32).min and c_max < np.iinfo(np.int32).max:\n",
    "                    df[col] = df[col].astype(np.int32)\n",
    "                elif c_min > np.iinfo(np.int64).min and c_max < np.iinfo(np.int64).max:\n",
    "                    df[col] = df[col].astype(np.int64)  \n",
    "            else:\n",
    "                if c_min > np.finfo(np.float16).min and c_max < np.finfo(np.float16).max:\n",
    "                    df[col] = df[col].astype(np.float16)\n",
    "                elif c_min > np.finfo(np.float32).min and c_max < np.finfo(np.float32).max:\n",
    "                    df[col] = df[col].astype(np.float32)\n",
    "                else:\n",
    "                    df[col] = df[col].astype(np.float64)    \n",
    "    end_mem = df.memory_usage().sum() / 1024**2\n",
    "    if verbose: print('Mem. usage decreased to {:5.2f} Mb ({:.1f}% reduction)'.format(end_mem, 100 * (start_mem - end_mem) / start_mem))\n",
    "    return df\n",
    "\n",
    "\n",
    "\n",
    "def timestamp_to_date (timestamp) :\n",
    "    # 获得当前时间时间戳\n",
    "    #转换为其他日期格式,如:\"%Y-%m-%d %H:%M:%S\"\n",
    "    timeArray = time.localtime(int(timestamp))\n",
    "    otherStyleTime = time.strftime(\"%Y-%m-%d %H:%M:%S\", timeArray)\n",
    "    return otherStyleTime\n",
    "\n",
    "def date_2_timestamp (date_time) :\n",
    "    # 字符类型的时间\n",
    "    # 转为时间数组\n",
    "    timeArray = time.strptime(date_time, \"%Y%m%d%H%M%S\")    \n",
    "    # 转为时间戳\n",
    "    timeStamp = np.int64(time.mktime(timeArray))\n",
    "    return timeStamp  # 1381419600\n",
    "\n",
    "def get_candidate_recall (recall_list, candidate_vid_set) :\n",
    "    result = []\n",
    "    for recall in recall_list :\n",
    "        if recall in candidate_vid_set :\n",
    "            result.append(recall)\n",
    "            \n",
    "    return result \n",
    "\n",
    "#候选的did-fvid\n",
    "def get_candidate_did_fvid (df_click, df_vid_info) :\n",
    "    candidate_did_fvid = df_click[['did', 'fvid']].drop_duplicates().reset_index(drop=True)\n",
    "    candidate_did_fvid = candidate_did_fvid.merge(df_vid_info[['vid', 'cid']].rename(columns = {'vid' : 'fvid'}), on='fvid', how='left')\n",
    "    return candidate_did_fvid\n",
    "\n",
    "#候选的vid\n",
    "def get_candidate_vid (df_click, df_vid_info) :\n",
    "    candidate_vid = df_click[['vid']].drop_duplicates().reset_index(drop=True)\n",
    "    candidate_vid = candidate_vid.merge(df_vid_info[['vid', 'cid', 'online_time']], on='vid', how='left')\n",
    "    return candidate_vid"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "aa3c0944-2ab3-4520-a603-63d8b9dffd4d",
   "metadata": {},
   "outputs": [],
   "source": [
    "df_click_data_set = []\n",
    "df_show_data_set = []\n",
    "df_main_vv_set = []\n",
    "\n",
    "base_path = \"/home/THLUO/competition/A_CSV/\"\n",
    "\n",
    "for part in tqdm_notebook(['part_1', 'part_2', 'part_3', 'part_4', 'part_5', 'part_6', 'part_7']) :\n",
    "    df_click_data = pd.read_csv(base_path + \"{}/dbfeed_click_info.csv\".format(part))\n",
    "    df_show_data = pd.read_csv(base_path + \"{}/dbfeed_show_info.csv\".format(part))\n",
    "    #df_main_vv = pd.read_pickle(\"/home/THLUO/competition/A2/{}/user_main_behavior.pkl\".format(part))\n",
    "    #del df_click_data['vid_hb']\n",
    "    #del df_click_data['duration']\n",
    "    #del df_click_data['cid']\n",
    "    df_click_data_set.append(df_click_data)\n",
    "    df_show_data_set.append(df_show_data)\n",
    "    \n",
    "    \n",
    "df_show_data = pd.concat(df_show_data_set).reset_index(drop=True)\n",
    "df_click_data = pd.concat(df_click_data_set).reset_index(drop=True)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "d5736f61-208a-4bae-ac6d-386f8e39df35",
   "metadata": {},
   "outputs": [],
   "source": [
    "#保存视频信息表\n",
    "df_vid_info = pd.read_csv(base_path + \"vid_info.csv\")\n",
    "#df_vid_info['online_time'] = df_vid_info['online_time'].apply(lambda x : np.NaN if x <= 0 else x)\n",
    "#保存视频明星表\n",
    "df_vid_tag_conf = pd.read_csv(base_path + \"vid_stars_info.csv\")\n",
    "#保存视频标签信息表\n",
    "df_vid_dim_tags_conf = pd.read_csv(base_path + \"vid_dim_tags_info.csv\")\n",
    "#保存标签信息表\n",
    "df_dim_tag_conf = pd.read_csv(base_path + \"dim_tags_info.csv\")\n",
    "\n",
    "df_click_data = df_click_data.merge(df_vid_info[['vid', 'cid']], on='vid', how='left')\n",
    "df_show_data = df_show_data.merge(df_vid_info[['vid', 'cid']], on='vid', how='left')\n",
    "\n",
    "df_show_data['date'] = df_show_data['time'].apply(timestamp_to_date).apply(lambda x : x.split(' ')[0])\n",
    "df_click_data['date'] = df_click_data['time'].apply(timestamp_to_date).apply(lambda x : x.split(' ')[0])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a07e3dec-ef29-4875-96ff-d7c7374ebfc6",
   "metadata": {},
   "outputs": [],
   "source": [
    "#训练集did, fvid候选集\n",
    "train_click = df_click_data[(df_click_data['date'] == '2021-03-25')].reset_index(drop=True)\n",
    "train_candidate_did_fvid = get_candidate_did_fvid(train_click, df_vid_info)\n",
    "#训练集vid候选集\n",
    "train_candidate_vid = get_candidate_vid(train_click, df_vid_info)\n",
    "\n",
    "#验证集did, fvid候选集\n",
    "valid_click = df_click_data[(df_click_data['date'] == '2021-03-26')].reset_index(drop=True)\n",
    "valid_candidate_did_fvid = get_candidate_did_fvid(valid_click, df_vid_info)\n",
    "#验证集vid候选集\n",
    "valid_candidate_vid = get_candidate_vid(valid_click, df_vid_info)\n",
    "\n",
    "#测试集did, fvid候选集\n",
    "test_candidate_did_fvid = pd.read_csv(base_path + \"/part_test/test_candidate_did_fvid.csv\")\n",
    "#测试集vid候选集\n",
    "test_candidate_vid = pd.read_csv(base_path + \"/part_test/test_candidate_vid.csv\")\n",
    "test_candidate_did_fvid = test_candidate_did_fvid.merge(df_vid_info[['vid', 'cid']].rename(columns={\"vid\" : \"fvid\"}), on='fvid', how='left')\n",
    "test_candidate_vid = test_candidate_vid.merge(df_vid_info[['vid', 'cid']], on='vid', how='left')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "e4696c9e-2b4c-450c-8f5c-d54f6ddfd16a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#过去N天fvid下的topN点击率进行召回\n",
    "def recall_by_fvid_topN_ctr (df_click, df_show, topN, recall_name, start_date, end_date) :\n",
    "    df_click = df_click[(df_click['date'] >= start_date) & (df_click['date'] < end_date)]\n",
    "    df_fvid_vid_clicks = df_click.groupby(['fvid', 'vid'])['pos'].count().reset_index().rename(columns={'pos' : 'click_counts'})\n",
    "    df_fvid_vid_clicks['click_rank'] = df_fvid_vid_clicks.groupby('fvid')['click_counts'].rank(method='dense', ascending=False)\n",
    "    \n",
    "    df_show = df_show[(df_show['date'] >= start_date) & (df_show['date'] < end_date)]\n",
    "    df_fvid_vid_shows = df_show.groupby(['fvid', 'vid'])['pos'].count().reset_index().rename(columns={'pos' : 'show_counts'})\n",
    "    df_fvid_vid_clicks = df_fvid_vid_clicks.merge(df_fvid_vid_shows, on=['fvid', 'vid'], how='left')\n",
    "    df_fvid_vid_clicks['vid_ctr'] = df_fvid_vid_clicks['click_counts'] / (df_fvid_vid_clicks['show_counts'])\n",
    "    df_fvid_vid_clicks['ctr_rank'] = df_fvid_vid_clicks.groupby('fvid')['vid_ctr'].rank(method='dense', ascending=False)    \n",
    "    \n",
    "    #过去N天，fvid下，点击率最高的N个视频\n",
    "    df_recall = df_fvid_vid_clicks[df_fvid_vid_clicks['ctr_rank'] <= topN]\n",
    "    df_recall = df_recall.groupby('fvid')['vid'].agg(lambda x : list(x)).reset_index().rename(columns={'vid' : recall_name})\n",
    "    \n",
    "    return df_recall\n",
    "\n",
    "#过去N天fvid下的topN观看比例进行召回\n",
    "def recall_by_fvid_topN_vts_ratio (df_click, topN, recall_name, start_date, end_date) :\n",
    "    df_click = df_click[(df_click['date'] >= start_date) & (df_click['date'] < end_date)]\n",
    "    df_fvid_vid_hb_ratio = df_click.groupby(['fvid', 'vid'])['vts_ratio'].sum().reset_index().rename(columns={'vts_ratio' : 'vid_sum_vts_ratio'})\n",
    "    df_fvid_vid_hb_ratio['vid_sum_vts_ratio_rank'] = df_fvid_vid_hb_ratio.groupby('fvid')['vid_sum_vts_ratio'].rank(method='dense', ascending=False)\n",
    "         \n",
    "    #过去N天，fvid下，观看时长最高的N个视频\n",
    "    df_recall = df_fvid_vid_hb_ratio[df_fvid_vid_hb_ratio['vid_sum_vts_ratio_rank'] <= topN]\n",
    "    df_recall = df_recall.groupby('fvid')['vid'].agg(lambda x : list(x)).reset_index().rename(columns={'vid' : recall_name})    \n",
    "    return df_recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "24feefe8-b87b-4484-b18c-0ed58ef9d5de",
   "metadata": {},
   "outputs": [],
   "source": [
    "def recall_by_history (df_click_data, df_show_data, candidate_vid, topN, start_date, end_date) :\n",
    "    topN = 10\n",
    "    candidate_vid_set = set(candidate_vid['vid'].unique())\n",
    "    #过去N天fvid下的topN点击率进行召回\n",
    "    df_recall_1 = recall_by_fvid_topN_ctr (df_click_data, df_show_data, topN, 'topN_fvid_vid_ctr', start_date, end_date)\n",
    "    #过去N天fvid下的观看比例进行召回\n",
    "    df_recall_2 = recall_by_fvid_topN_vts_ratio (df_click_data, topN, 'topN_fvid_vid_vts_ratio', start_date, end_date)\n",
    "    \n",
    "    #合并\n",
    "    df_recall = df_recall_1.merge(df_recall_2, on='fvid', how='outer')\n",
    "\n",
    "    df_recall['topN_fvid_vid_ctr'] = df_recall['topN_fvid_vid_ctr'].fillna(0).apply(lambda x : [] if x == 0 else x)\n",
    "    df_recall['topN_fvid_vid_vts_ratio'] = df_recall['topN_fvid_vid_vts_ratio'].fillna(0).apply(lambda x : [] if x == 0 else x)\n",
    "\n",
    "    #取并集\n",
    "    df_recall['recall_list'] = (df_recall['topN_fvid_vid_ctr'] + df_recall['topN_fvid_vid_vts_ratio'] ).apply(set).apply(list)\n",
    "\n",
    "    #只选取候选vid中的视频\n",
    "    df_recall['recall_list'] = df_recall['recall_list'].apply(get_candidate_recall, args=(candidate_vid_set, ))\n",
    "    return df_recall"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "80d780ca-80b7-4903-98cc-da342034b828",
   "metadata": {},
   "outputs": [],
   "source": [
    "#历史N天的热门(点击次数，点击率，观看时长，观看比例)\n",
    "def recall_by_hot_data (df_click, df_show, candidate_vid, hotN, start_date, end_date) :\n",
    "    df_click = df_click[(df_click['date'] >= start_date) & (df_click['date'] < end_date)]\n",
    "    #过去N天，fvid下，vid的点击次数的排序\n",
    "    df_fvid_vid_clicks = df_click.groupby(['fvid', 'vid'])['pos'].count().reset_index().rename(columns={'pos' : 'click_counts'})\n",
    "    \n",
    "    #过去N天，fvid下，vid的观看比例的排序\n",
    "    df_fvid_vid_vts_ratio = df_click.groupby(['fvid', 'vid'])['vts_ratio'].sum().reset_index().rename(columns={'vts_ratio' : 'vid_sum_vts_ratio'})\n",
    "\n",
    "    #候选视频中历史点击次数最高的N个视频来填补\n",
    "    df_top_click_vid = df_fvid_vid_clicks.groupby('vid')['click_counts'].sum().reset_index().sort_values('click_counts', ascending=False).reset_index(drop=True)\n",
    "    df_top_click_vid = df_top_click_vid.merge(candidate_vid, on='vid', how='inner')\n",
    "    #候选视频中历史曝光观看时长比例最高的N个视频来填补\n",
    "    df_top_vts_ratio_vid = df_fvid_vid_vts_ratio.groupby('vid')['vid_sum_vts_ratio'].sum().reset_index().sort_values('vid_sum_vts_ratio', ascending=False).reset_index(drop=True)\n",
    "    df_top_vts_ratio_vid = df_top_vts_ratio_vid.merge(candidate_vid, on='vid', how='inner')\n",
    "    #取并集\n",
    "    hot_vid_recall = list(df_top_click_vid['vid'].values[:hotN])+ list(df_top_vts_ratio_vid['vid'].values[:hotN])\n",
    "    hot_vid_recall = list(set(hot_vid_recall))    \n",
    "    return hot_vid_recall\n",
    "\n",
    "def recall_by_hot (train_data, df_click_data, df_show_data, candidate_vid, start_date, end_date, hotN) :\n",
    "    #增加过去N天的topN热门视频\n",
    "    hot_vid_recall = recall_by_hot_data (df_click_data, df_show_data, candidate_vid, hotN, start_date, end_date)\n",
    "    #增加历史热门视频作为召回\n",
    "    train_data['recall_list'] = train_data['recall_list'].apply(lambda x : list(set(x + hot_vid_recall)))\n",
    "    return train_data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "4f50c0ae-fa26-43e2-8247-e3e25af39288",
   "metadata": {},
   "outputs": [],
   "source": [
    "def recall (candidate_vid, candidate_did_fvid, df_click, df_show, start_date, end_date, topN = 10) :\n",
    "    #过N天fvid下，vid的点击率，观看比例的TOPN进行召回\n",
    "    df_recall = recall_by_history(df_click, df_show, candidate_vid, topN, start_date, end_date)\n",
    "\n",
    "    data_recall_list = candidate_did_fvid.merge(df_recall, on='fvid', how='left')\n",
    "    data_recall_list['recall_list'] = data_recall_list['recall_list'].apply(lambda x : [] if isNan_2(x) else x)\n",
    "    \n",
    "    #过N天的热门(vid的点击次数，观看比例的TOPN进行召回)\n",
    "    data_recall_list = recall_by_hot (data_recall_list, df_click, df_show, candidate_vid, start_date, end_date, 20)\n",
    "\n",
    "    data = explode_df(data_recall_list)\n",
    "    df_click['label'] = 1\n",
    "    data = data.merge(df_click[['did', 'fvid', 'vid', 'label']], on=['did', 'fvid', 'vid'], how='left')\n",
    "    data['label'].fillna(0, inplace=True)\n",
    "    \n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "fe184d81-7143-415a-85c2-953be9fe94ec",
   "metadata": {},
   "outputs": [],
   "source": [
    "def explode_df(train_data) :\n",
    "    did_list = []\n",
    "    fvid_list = []\n",
    "    vid_list = []\n",
    "    for row in train_data[['did', 'fvid', 'recall_list']].values :\n",
    "        did = row[0]\n",
    "        fvid = row[1]\n",
    "        recall_list = row[2]\n",
    "        for recall in recall_list :\n",
    "            did_list.append(did)\n",
    "            fvid_list.append(fvid)\n",
    "            vid_list.append(recall)\n",
    "\n",
    "    df = pd.DataFrame()\n",
    "    df['did'] = did_list\n",
    "    df['fvid'] = fvid_list\n",
    "    df['vid'] = vid_list\n",
    "    return df "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "ac09cb43-57a8-4c83-982b-d294143f948c",
   "metadata": {},
   "outputs": [],
   "source": [
    "#召回构造训练样本\n",
    "data_train = recall(train_candidate_vid, train_candidate_did_fvid, df_click_data, df_show_data, \"2021-03-20\", \"2021-03-25\")\n",
    "data_valid = recall(valid_candidate_vid, valid_candidate_did_fvid, df_click_data, df_show_data, \"2021-03-21\", \"2021-03-26\")\n",
    "data_test = recall(test_candidate_vid, test_candidate_did_fvid, df_click_data, df_show_data, \"2021-03-22\", \"2021-03-27\")"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "5d35c532-267b-4344-9a28-acdb46ca0182",
   "metadata": {},
   "outputs": [],
   "source": [
    "def get_user_click_feat(df_click, key, start_date, end_date) :\n",
    "    df_feats = df_click.groupby('did')[key].nunique()\n",
    "    return df_feats\n",
    "\n",
    "def get_user_show_feat(df_show, key, start_date, end_date) :\n",
    "    df_feats = df_show.groupby('did')[key].nunique()\n",
    "    return df_feats\n",
    "\n",
    "def get_fvid_click_feats (df_click, key, start_date, end_date) :\n",
    "    df_feats = df_click.groupby('fvid')[key].nunique()\n",
    "    return df_feats    \n",
    "\n",
    "def get_fvid_show_feats (df_show, key, start_date, end_date) :\n",
    "    df_feats = df_show.groupby('fvid')[key].nunique()\n",
    "    return df_feats \n",
    "\n",
    "def get_vid_click_feats (df_click, key, start_date, end_date) :\n",
    "    df_feats = df_click.groupby('vid')[key].nunique()\n",
    "    return df_feats \n",
    "\n",
    "def get_vid_show_feats (df_show, key, start_date, end_date) :\n",
    "    df_feats = df_show.groupby('vid')[key].nunique()\n",
    "    return df_feats\n",
    "\n",
    "def get_cid_click_feats (df_click, key, start_date, end_date) :\n",
    "    df_feats = df_click.groupby('cid')[key].nunique()\n",
    "    return df_feats \n",
    "\n",
    "def get_cid_show_feats (df_show, key, start_date, end_date) :\n",
    "    df_feats = df_show.groupby('cid')[key].nunique()\n",
    "    return df_feats\n",
    "\n",
    "def get_user_cross_click_feats (df_click, key, start_date, end_date) :\n",
    "    df_feats = df_click.groupby(['did', key])['pos'].count().reset_index().rename(columns = {'pos' : 'did_{}_click_counts'.format(key)})   \n",
    "    return df_feats\n",
    "\n",
    "def get_fvid_cross_click_feats (df_click, key, start_date, end_date) :\n",
    "    df_feats = df_click.groupby(['fvid', key])['pos'].count().reset_index().rename(columns = {'pos' : 'fvid_{}_click_counts'.format(key)})   \n",
    "    return df_feats   "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "b3ef40ec-fd7f-49f1-934a-9593a9185342",
   "metadata": {},
   "outputs": [],
   "source": [
    "def make_features (data, candidate_did_fvid, start_date, end_date) :\n",
    "    data = data.merge(df_vid_info, on='vid', how='left')\n",
    "    \n",
    "    df_click = df_click_data[(df_click_data['date'] >= start_date) & (df_click_data['date'] < end_date)]\n",
    "    df_show = df_show_data[(df_show_data['date'] >= start_date) & (df_show_data['date'] < end_date)]\n",
    "    \n",
    "    #历史上用户点击过多少fvid\n",
    "    df_did_click_unique_fvid = get_user_click_feat(df_click, 'fvid', start_date, end_date)\n",
    "    data['did_click_unique_fvid'] = data['did'].map(df_did_click_unique_fvid)\n",
    "    #历史上用户点击过多少vid\n",
    "    df_did_click_unique_vid = get_user_click_feat(df_click, 'vid', start_date, end_date)\n",
    "    data['did_click_unique_vid'] = data['did'].map(df_did_click_unique_vid)\n",
    "    #历史上用户点击过多少cid\n",
    "    df_did_click_unique_cid = get_user_click_feat(df_click, 'cid', start_date, end_date)\n",
    "    data['did_click_unique_cid'] = data['did'].map(df_did_click_unique_cid)\n",
    "    #历史上用户曝光过多少vid\n",
    "    df_did_show_unique_vid = get_user_show_feat(df_show, 'vid', start_date, end_date)\n",
    "    data['did_show_unique_vid'] = data['did'].map(df_did_show_unique_vid)\n",
    "    #历史上用户曝光过多少cid\n",
    "    df_did_show_unique_cid = get_user_show_feat(df_show, 'cid', start_date, end_date)\n",
    "    data['did_show_unique_cid'] = data['did'].map(df_did_show_unique_cid)\n",
    "\n",
    "    #历史上fvid被多少用户点击过\n",
    "    df_fvid_click_unique_did = get_fvid_click_feats(df_click, 'did', start_date, end_date)\n",
    "    data['fvid_click_unique_did'] = data['fvid'].map(df_fvid_click_unique_did)\n",
    "    #历史上fvid对多少用户曝光过\n",
    "    df_fvid_show_unique_did = get_fvid_show_feats(df_show, 'did', start_date, end_date) \n",
    "    data['fvid_show_unique_did'] = data['fvid'].map(df_fvid_show_unique_did)\n",
    "\n",
    "    #历史上vid被多少用户点击过\n",
    "    df_vid_click_unique_did = get_vid_click_feats(df_click, 'did', start_date, end_date)\n",
    "    data['vid_click_unique_did'] = data['vid'].map(df_vid_click_unique_did)\n",
    "    #历史上vid被多少用户点击过\n",
    "    df_vid_show_unique_did = get_vid_show_feats(df_show, 'did', start_date, end_date) \n",
    "    data['vid_show_unique_did'] = data['vid'].map(df_vid_show_unique_did)\n",
    "\n",
    "    #历史上cid被多少用户点击过\n",
    "    df_cid_click_unique_did = get_cid_click_feats(df_click, 'did', start_date, end_date) \n",
    "    data['cid_click_unique_did'] = data['cid'].map(df_cid_click_unique_did)\n",
    "    #历史上cid对多少用户曝光过\n",
    "    df_cid_show_unique_did = get_cid_show_feats(df_show, 'did', start_date, end_date) \n",
    "    data['cid_show_unique_did'] = data['cid'].map(df_cid_show_unique_did)\n",
    "\n",
    "    #历史上用户点击fvid的次数\n",
    "    df_did_fvid_clicks = get_user_cross_click_feats(df_click, 'fvid', start_date, end_date)\n",
    "    data = data.merge(df_did_fvid_clicks, on=['did', 'fvid'], how='left')\n",
    "    #历史上用户点击vid的次数\n",
    "    df_did_vid_clicks = get_user_cross_click_feats(df_click, 'vid', start_date, end_date)\n",
    "    data = data.merge(df_did_vid_clicks, on=['did', 'vid'], how='left')\n",
    "    #历史上用户点击cid的次数\n",
    "    df_did_cid_clicks = get_user_cross_click_feats(df_click, 'cid', start_date, end_date)\n",
    "    data = data.merge(df_did_cid_clicks, on=['did', 'cid'], how='left')\n",
    "\n",
    "    #历史上fvid下，vid的点击次数\n",
    "    df_fvid_vid_clicks = get_fvid_cross_click_feats(df_click, 'vid', start_date, end_date)\n",
    "    data = data.merge(df_fvid_vid_clicks, on=['fvid', 'vid'], how='left')\n",
    "    #历史上fvid下，cid的点击次数\n",
    "    df_fvid_cid_clicks = get_fvid_cross_click_feats(df_click, 'cid', start_date, end_date)\n",
    "    data = data.merge(df_fvid_cid_clicks, on=['fvid', 'cid'], how='left')\n",
    "    \n",
    "    #当天用户观看候选合集的次数\n",
    "    df_current_did_cid_clicks = candidate_did_fvid.groupby(['did', 'cid'])['fvid'].count().reset_index().rename(columns = {'fvid' : 'current_did_cid_clicks'})\n",
    "    data = data.merge(df_current_did_cid_clicks, on=['did', 'cid'], how='left')\n",
    "    \n",
    "    data.fillna(0, inplace=True)\n",
    "\n",
    "    return data"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "224f02a0-e1ce-4b11-92b7-d98c1f0255c8",
   "metadata": {},
   "outputs": [],
   "source": [
    "get_ipython().run_cell_magic('time', '', '\\nprint (\"提取训练集的特征\")\\ntrain_data = make_features(data_train, train_candidate_did_fvid, \"2021-03-20\", \"2021-03-25\")\\nprint (\"提取验证集的特征\")\\nvalid_data = make_features(data_valid, valid_candidate_did_fvid, \"2021-03-21\", \"2021-03-26\")\\nprint (\"提取测试集的特征\")\\ntest_data = make_features(data_test, test_candidate_did_fvid, \"2021-03-22\", \"2021-03-27\")\\n\\ntrain_data = reduce_mem_usage(train_data)\\nvalid_data = reduce_mem_usage(valid_data)\\ntest_data = reduce_mem_usage(test_data)')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "247eb09c-ac2a-4454-80df-4a570652e74e",
   "metadata": {},
   "outputs": [],
   "source": [
    "useless_cols = ['label', 'did', 'online_time', 'preds', 'preds_rank', 'vts_ratio', 'vid_vts', 'key_word']\n",
    "features = train_data.columns[~train_data.columns.isin(useless_cols)].values\n",
    "print (features)\n",
    "\n",
    "\n",
    "params = {\n",
    "    'objective': 'binary', #定义的目标函数\n",
    "    'metric': {'auc'},\n",
    "    'boosting_type' : 'gbdt',\n",
    "\n",
    "    'learning_rate': 0.05,\n",
    "    'max_depth' : 12,\n",
    "    'num_leaves' : 2 ** 6,\n",
    "\n",
    "    'min_child_weight' : 10,\n",
    "    'min_data_in_leaf' : 40,\n",
    "\n",
    "    'feature_fraction' : 0.70,\n",
    "    'subsample' : 0.75,\n",
    "    'seed' : 114,\n",
    "\n",
    "    'nthread' : -1,\n",
    "    'bagging_freq' : 1,\n",
    "    'verbose' : -1,\n",
    "    #'scale_pos_weight':200\n",
    "}\n",
    "\n",
    "\n",
    "trn_data = lgb.Dataset(train_data[features], label=train_data['label'].values)\n",
    "val_data = lgb.Dataset(valid_data[features], label=valid_data['label'].values)\n",
    "print (\"train_data : \", train_data.info())\n",
    "print (\"valid_data : \", valid_data.info())\n",
    "clf = lgb.train(params,\n",
    "                trn_data,\n",
    "                3000,\n",
    "                valid_sets=[trn_data, val_data],\n",
    "                verbose_eval=50,\n",
    "                early_stopping_rounds=50)#, feval=self_gauc)\n",
    "\n",
    "\n",
    "valid_data['preds'] = clf.predict(valid_data[features], num_iteration=clf.best_iteration)\n",
    "valid_data = valid_data.sort_values(by=['did', 'fvid', 'preds'], ascending=False).reset_index(drop=True)\n",
    "valid_data['preds_rank'] = valid_data.groupby(['did', 'fvid'])['vid'].cumcount() + 1\n",
    "\n",
    "valid_solution = valid_data[valid_data['preds_rank'] <= 6][['did', 'fvid', 'vid']]\n",
    "valid_solution['vts_ratio'] = 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "a7b36b83-59ab-4ce6-be50-479b7ca70749",
   "metadata": {},
   "outputs": [],
   "source": [
    "def AP_N (actual_vids, pred_vids, N) :\n",
    "    if len(pred_vids) < N :\n",
    "        return 0\n",
    "    down = np.min([len(actual_vids), N])\n",
    "    actual_vids = set(actual_vids)\n",
    "    up,flag,correct = 0,0,0\n",
    "    for pv in pred_vids :\n",
    "        if flag > N :\n",
    "            break\n",
    "        flag += 1\n",
    "        if pv in actual_vids :\n",
    "            correct += 1\n",
    "            up += float(correct) / flag\n",
    "    \n",
    "    return float(up) / down \n",
    "\n",
    "\n",
    "#MAP@6评分\n",
    "def cal_map (df_answer, df_solution) :\n",
    "    df_A_map_6 = df_answer.groupby(['did', 'fvid'])['vid'].apply(list).reset_index()\n",
    "    test_solution_map_6 = df_solution.groupby(['did', 'fvid'])['vid'].apply(list).reset_index()\n",
    "    test_solution_map_6.rename(columns = {'vid' : 'pred_vid'}, inplace=True)\n",
    "    df_A_map_6 = df_A_map_6.merge(test_solution_map_6, on=['did', 'fvid'], how='left')    \n",
    "    return df_A_map_6.apply(lambda x : AP_N(x[\"vid\"], x[\"pred_vid\"], 6),axis=1).mean()\n",
    "\n",
    "\n",
    "#Task2评分\n",
    "def cal_task2_score (df_answer, df_solution) :\n",
    "    test_t2 = df_answer.merge(df_solution, on=['did', 'fvid', 'vid'], how='left')\n",
    "    test_t2 = test_t2.rename(columns = {'vts_ratio_x' : 'actual_vts_ratio'}).rename(columns = {'vts_ratio_y' : 'pred_vts_ratio'})\n",
    "    df_score = test_t2[test_t2['pred_vts_ratio'].notnull()].reset_index(drop=True)\n",
    "    df_score['T2_Score'] = 1 / (1 + np.sqrt(np.abs(df_score['actual_vts_ratio'] - df_score['pred_vts_ratio'])))\n",
    "    df_score = df_score.groupby(['did', 'fvid'])['T2_Score'].sum().reset_index()\n",
    "    df_temp = df_answer.groupby(['did', 'fvid'])['vid'].count().reset_index()\n",
    "    df_score = df_temp.merge(df_score, on=['did', 'fvid'], how='left')\n",
    "    df_score['T2_Score'].fillna(0, inplace=True)\n",
    "    df_score['T2_Score'] = df_score['T2_Score'] / df_score['vid']\n",
    "    S = len(df_answer.drop_duplicates(['did', 'fvid']))\n",
    "    t2_score = float(df_score['T2_Score'].sum()) / float(S)\n",
    "    return t2_score "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "8b3129a9-567d-4e41-984d-ee02f05bbc1a",
   "metadata": {},
   "outputs": [],
   "source": [
    "#验证集的分数\n",
    "df_valid_answer = valid_click.groupby(['did', 'fvid', 'vid'])['vts_ratio'].sum().reset_index()\n",
    "df_valid_answer['vts_ratio'] = df_valid_answer['vts_ratio'].apply(lambda x : 1 if x >= 1 else x).apply(lambda x : 0 if x <= 0 else x)\n",
    "map_6 = cal_map(df_valid_answer, valid_solution)\n",
    "task_2 = cal_task2_score(df_valid_answer, valid_solution)\n",
    "print (map_6 * 0.7 + task_2 * 0.3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "dbf7a496-8c2c-40ca-aff1-7035ae5ca64d",
   "metadata": {},
   "outputs": [],
   "source": [
    "useless_cols = ['label', 'did', 'online_time', 'preds', 'preds_rank', 'vts_ratio', 'vid_vts', 'key_word']\n",
    "features = valid_data.columns[~valid_data.columns.isin(useless_cols)].values\n",
    "print (features)\n",
    "\n",
    "\n",
    "params = {\n",
    "    'objective': 'binary', #定义的目标函数\n",
    "    'metric': {'auc'},\n",
    "    'boosting_type' : 'gbdt',\n",
    "\n",
    "    'learning_rate': 0.05,\n",
    "    'max_depth' : 12,\n",
    "    'num_leaves' : 2 ** 6,\n",
    "\n",
    "    'min_child_weight' : 10,\n",
    "    'min_data_in_leaf' : 40,\n",
    "\n",
    "    'feature_fraction' : 0.70,\n",
    "    'subsample' : 0.75,\n",
    "    'seed' : 114,\n",
    "\n",
    "    'nthread' : -1,\n",
    "    'bagging_freq' : 1,\n",
    "    'verbose' : -1,\n",
    "    #'scale_pos_weight':200\n",
    "}\n",
    "\n",
    "\n",
    "trn_data = lgb.Dataset(valid_data[features], label=valid_data['label'].values)\n",
    "print (\"train_data : \", train_data.info())\n",
    "clf = lgb.train(params,\n",
    "                trn_data,\n",
    "                clf.best_iteration,\n",
    "                valid_sets=[trn_data],\n",
    "                verbose_eval=50,\n",
    "                early_stopping_rounds=50)#, feval=self_gauc)\n",
    "\n",
    "\n",
    "\n",
    "test_data['preds'] = clf.predict(test_data[features], num_iteration=clf.best_iteration)\n",
    "test_data = test_data.sort_values(by=['did', 'fvid', 'preds'], ascending=False).reset_index(drop=True)\n",
    "test_data['preds_rank'] = test_data.groupby(['did', 'fvid'])['vid'].cumcount() + 1\n",
    "\n",
    "test_solution = test_data[test_data['preds_rank'] <= 6][['did', 'fvid', 'vid']]\n",
    "test_solution['vts_ratio'] = 1\n",
    "\n",
    "test_solution.to_csv('test_solution.csv', index=None)"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
